{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Examples and Exercises from Think Stats, 2nd Edition\n",
    "\n",
    "http://thinkstats2.com\n",
    "\n",
    "Copyright 2016 Allen B. Downey\n",
    "\n",
    "MIT License: https://opensource.org/licenses/MIT\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import thinkstats2\n",
    "import thinkplot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multiple regression\n",
    "\n",
    "Let's load up the NSFG data again."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import first\n",
    "\n",
    "live, firsts, others = first.MakeFrames()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here's birth weight as a function of mother's age (which we saw in the previous chapter)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>       <td>totalwgt_lb</td>   <th>  R-squared:         </th> <td>   0.005</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.005</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   43.02</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Thu, 28 Feb 2019</td> <th>  Prob (F-statistic):</th> <td>5.72e-11</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>09:58:53</td>     <th>  Log-Likelihood:    </th> <td> -15897.</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>  9038</td>      <th>  AIC:               </th> <td>3.180e+04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>  9036</td>      <th>  BIC:               </th> <td>3.181e+04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     1</td>      <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "      <td></td>         <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th> <td>    6.8304</td> <td>    0.068</td> <td>  100.470</td> <td> 0.000</td> <td>    6.697</td> <td>    6.964</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>agepreg</th>   <td>    0.0175</td> <td>    0.003</td> <td>    6.559</td> <td> 0.000</td> <td>    0.012</td> <td>    0.023</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>1024.052</td> <th>  Durbin-Watson:     </th> <td>   1.618</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th>  <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td>3081.833</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>           <td>-0.601</td>  <th>  Prob(JB):          </th> <td>    0.00</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>       <td> 5.596</td>  <th>  Cond. No.          </th> <td>    118.</td>\n",
       "</tr>\n",
       "</table><br/><br/>Warnings:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:            totalwgt_lb   R-squared:                       0.005\n",
       "Model:                            OLS   Adj. R-squared:                  0.005\n",
       "Method:                 Least Squares   F-statistic:                     43.02\n",
       "Date:                Thu, 28 Feb 2019   Prob (F-statistic):           5.72e-11\n",
       "Time:                        09:58:53   Log-Likelihood:                -15897.\n",
       "No. Observations:                9038   AIC:                         3.180e+04\n",
       "Df Residuals:                    9036   BIC:                         3.181e+04\n",
       "Df Model:                           1                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "==============================================================================\n",
       "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
       "------------------------------------------------------------------------------\n",
       "Intercept      6.8304      0.068    100.470      0.000       6.697       6.964\n",
       "agepreg        0.0175      0.003      6.559      0.000       0.012       0.023\n",
       "==============================================================================\n",
       "Omnibus:                     1024.052   Durbin-Watson:                   1.618\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):             3081.833\n",
       "Skew:                          -0.601   Prob(JB):                         0.00\n",
       "Kurtosis:                       5.596   Cond. No.                         118.\n",
       "==============================================================================\n",
       "\n",
       "Warnings:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "\"\"\""
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import statsmodels.formula.api as smf\n",
    "\n",
    "formula = 'totalwgt_lb ~ agepreg'\n",
    "model = smf.ols(formula, data=live)\n",
    "results = model.fit()\n",
    "results.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can extract the parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6.830396973311047, 0.017453851471802877)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inter = results.params['Intercept']\n",
    "slope = results.params['agepreg']\n",
    "inter, slope"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And the p-value of the slope estimate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.722947107312786e-11"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "slope_pvalue = results.pvalues['agepreg']\n",
    "slope_pvalue"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And the coefficient of determination."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.004738115474710369"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.rsquared"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The difference in birth weight between first babies and others."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.12476118453549034"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diff_weight = firsts.totalwgt_lb.mean() - others.totalwgt_lb.mean()\n",
    "diff_weight"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The difference in age between mothers of first babies and others."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-3.5864347661500275"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diff_age = firsts.agepreg.mean() - others.agepreg.mean()\n",
    "diff_age"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The age difference plausibly explains about half of the difference in weight."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.06259709972169267"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "slope * diff_age"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Running a single regression with a categorical variable, `isfirst`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>       <td>totalwgt_lb</td>   <th>  R-squared:         </th> <td>   0.002</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.002</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   17.74</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Thu, 28 Feb 2019</td> <th>  Prob (F-statistic):</th> <td>2.55e-05</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>09:58:53</td>     <th>  Log-Likelihood:    </th> <td> -15909.</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>  9038</td>      <th>  AIC:               </th> <td>3.182e+04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>  9036</td>      <th>  BIC:               </th> <td>3.184e+04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     1</td>      <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "         <td></td>            <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>       <td>    7.3259</td> <td>    0.021</td> <td>  356.007</td> <td> 0.000</td> <td>    7.286</td> <td>    7.366</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>isfirst[T.True]</th> <td>   -0.1248</td> <td>    0.030</td> <td>   -4.212</td> <td> 0.000</td> <td>   -0.183</td> <td>   -0.067</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>988.919</td> <th>  Durbin-Watson:     </th> <td>   1.613</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th> <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td>2897.107</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>          <td>-0.589</td>  <th>  Prob(JB):          </th> <td>    0.00</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>      <td> 5.511</td>  <th>  Cond. No.          </th> <td>    2.58</td>\n",
       "</tr>\n",
       "</table><br/><br/>Warnings:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:            totalwgt_lb   R-squared:                       0.002\n",
       "Model:                            OLS   Adj. R-squared:                  0.002\n",
       "Method:                 Least Squares   F-statistic:                     17.74\n",
       "Date:                Thu, 28 Feb 2019   Prob (F-statistic):           2.55e-05\n",
       "Time:                        09:58:53   Log-Likelihood:                -15909.\n",
       "No. Observations:                9038   AIC:                         3.182e+04\n",
       "Df Residuals:                    9036   BIC:                         3.184e+04\n",
       "Df Model:                           1                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "===================================================================================\n",
       "                      coef    std err          t      P>|t|      [0.025      0.975]\n",
       "-----------------------------------------------------------------------------------\n",
       "Intercept           7.3259      0.021    356.007      0.000       7.286       7.366\n",
       "isfirst[T.True]    -0.1248      0.030     -4.212      0.000      -0.183      -0.067\n",
       "==============================================================================\n",
       "Omnibus:                      988.919   Durbin-Watson:                   1.613\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):             2897.107\n",
       "Skew:                          -0.589   Prob(JB):                         0.00\n",
       "Kurtosis:                       5.511   Cond. No.                         2.58\n",
       "==============================================================================\n",
       "\n",
       "Warnings:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "\"\"\""
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "live['isfirst'] = live.birthord == 1\n",
    "formula = 'totalwgt_lb ~ isfirst'\n",
    "results = smf.ols(formula, data=live).fit()\n",
    "results.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now finally running a multiple regression:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>       <td>totalwgt_lb</td>   <th>  R-squared:         </th> <td>   0.005</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.005</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   24.02</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Thu, 28 Feb 2019</td> <th>  Prob (F-statistic):</th> <td>3.95e-11</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>09:58:53</td>     <th>  Log-Likelihood:    </th> <td> -15894.</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>  9038</td>      <th>  AIC:               </th> <td>3.179e+04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>  9035</td>      <th>  BIC:               </th> <td>3.182e+04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     2</td>      <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "         <td></td>            <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>       <td>    6.9142</td> <td>    0.078</td> <td>   89.073</td> <td> 0.000</td> <td>    6.762</td> <td>    7.066</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>isfirst[T.True]</th> <td>   -0.0698</td> <td>    0.031</td> <td>   -2.236</td> <td> 0.025</td> <td>   -0.131</td> <td>   -0.009</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>agepreg</th>         <td>    0.0154</td> <td>    0.003</td> <td>    5.499</td> <td> 0.000</td> <td>    0.010</td> <td>    0.021</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>1019.945</td> <th>  Durbin-Watson:     </th> <td>   1.618</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th>  <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td>3063.682</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>           <td>-0.599</td>  <th>  Prob(JB):          </th> <td>    0.00</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>       <td> 5.588</td>  <th>  Cond. No.          </th> <td>    137.</td>\n",
       "</tr>\n",
       "</table><br/><br/>Warnings:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:            totalwgt_lb   R-squared:                       0.005\n",
       "Model:                            OLS   Adj. R-squared:                  0.005\n",
       "Method:                 Least Squares   F-statistic:                     24.02\n",
       "Date:                Thu, 28 Feb 2019   Prob (F-statistic):           3.95e-11\n",
       "Time:                        09:58:53   Log-Likelihood:                -15894.\n",
       "No. Observations:                9038   AIC:                         3.179e+04\n",
       "Df Residuals:                    9035   BIC:                         3.182e+04\n",
       "Df Model:                           2                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "===================================================================================\n",
       "                      coef    std err          t      P>|t|      [0.025      0.975]\n",
       "-----------------------------------------------------------------------------------\n",
       "Intercept           6.9142      0.078     89.073      0.000       6.762       7.066\n",
       "isfirst[T.True]    -0.0698      0.031     -2.236      0.025      -0.131      -0.009\n",
       "agepreg             0.0154      0.003      5.499      0.000       0.010       0.021\n",
       "==============================================================================\n",
       "Omnibus:                     1019.945   Durbin-Watson:                   1.618\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):             3063.682\n",
       "Skew:                          -0.599   Prob(JB):                         0.00\n",
       "Kurtosis:                       5.588   Cond. No.                         137.\n",
       "==============================================================================\n",
       "\n",
       "Warnings:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "\"\"\""
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "formula = 'totalwgt_lb ~ isfirst + agepreg'\n",
    "results = smf.ols(formula, data=live).fit()\n",
    "results.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As expected, when we control for mother's age, the apparent difference due to `isfirst` is cut in half.\n",
    "\n",
    "If we add age squared, we can control for a quadratic relationship between age and weight."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>       <td>totalwgt_lb</td>   <th>  R-squared:         </th> <td>   0.007</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.007</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   22.64</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Thu, 28 Feb 2019</td> <th>  Prob (F-statistic):</th> <td>1.35e-14</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>09:58:54</td>     <th>  Log-Likelihood:    </th> <td> -15884.</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>  9038</td>      <th>  AIC:               </th> <td>3.178e+04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>  9034</td>      <th>  BIC:               </th> <td>3.181e+04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     3</td>      <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "         <td></td>            <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>       <td>    5.6923</td> <td>    0.286</td> <td>   19.937</td> <td> 0.000</td> <td>    5.133</td> <td>    6.252</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>isfirst[T.True]</th> <td>   -0.0504</td> <td>    0.031</td> <td>   -1.602</td> <td> 0.109</td> <td>   -0.112</td> <td>    0.011</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>agepreg</th>         <td>    0.1124</td> <td>    0.022</td> <td>    5.113</td> <td> 0.000</td> <td>    0.069</td> <td>    0.155</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>agepreg2</th>        <td>   -0.0018</td> <td>    0.000</td> <td>   -4.447</td> <td> 0.000</td> <td>   -0.003</td> <td>   -0.001</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>1007.149</td> <th>  Durbin-Watson:     </th> <td>   1.616</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th>  <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td>3003.343</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>           <td>-0.594</td>  <th>  Prob(JB):          </th> <td>    0.00</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>       <td> 5.562</td>  <th>  Cond. No.          </th> <td>1.39e+04</td>\n",
       "</tr>\n",
       "</table><br/><br/>Warnings:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.<br/>[2] The condition number is large, 1.39e+04. This might indicate that there are<br/>strong multicollinearity or other numerical problems."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:            totalwgt_lb   R-squared:                       0.007\n",
       "Model:                            OLS   Adj. R-squared:                  0.007\n",
       "Method:                 Least Squares   F-statistic:                     22.64\n",
       "Date:                Thu, 28 Feb 2019   Prob (F-statistic):           1.35e-14\n",
       "Time:                        09:58:54   Log-Likelihood:                -15884.\n",
       "No. Observations:                9038   AIC:                         3.178e+04\n",
       "Df Residuals:                    9034   BIC:                         3.181e+04\n",
       "Df Model:                           3                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "===================================================================================\n",
       "                      coef    std err          t      P>|t|      [0.025      0.975]\n",
       "-----------------------------------------------------------------------------------\n",
       "Intercept           5.6923      0.286     19.937      0.000       5.133       6.252\n",
       "isfirst[T.True]    -0.0504      0.031     -1.602      0.109      -0.112       0.011\n",
       "agepreg             0.1124      0.022      5.113      0.000       0.069       0.155\n",
       "agepreg2           -0.0018      0.000     -4.447      0.000      -0.003      -0.001\n",
       "==============================================================================\n",
       "Omnibus:                     1007.149   Durbin-Watson:                   1.616\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):             3003.343\n",
       "Skew:                          -0.594   Prob(JB):                         0.00\n",
       "Kurtosis:                       5.562   Cond. No.                     1.39e+04\n",
       "==============================================================================\n",
       "\n",
       "Warnings:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "[2] The condition number is large, 1.39e+04. This might indicate that there are\n",
       "strong multicollinearity or other numerical problems.\n",
       "\"\"\""
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "live['agepreg2'] = live.agepreg**2\n",
    "formula = 'totalwgt_lb ~ isfirst + agepreg + agepreg2'\n",
    "results = smf.ols(formula, data=live).fit()\n",
    "results.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When we do that, the apparent effect of `isfirst` gets even smaller, and is no longer statistically significant.\n",
    "\n",
    "These results suggest that the apparent difference in weight between first babies and others might be explained by difference in mothers' ages, at least in part."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Mining\n",
    "\n",
    "We can use `join` to combine variables from the preganancy and respondent tables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nsfg\n",
    "\n",
    "live = live[live.prglngth>30]\n",
    "resp = nsfg.ReadFemResp()\n",
    "resp.index = resp.caseid\n",
    "join = live.join(resp, on='caseid', rsuffix='_r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And we can search for variables with explanatory power.\n",
    "\n",
    "Because we don't clean most of the variables, we are probably missing some good ones."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import patsy\n",
    "\n",
    "def GoMining(df):\n",
    "    \"\"\"Searches for variables that predict birth weight.\n",
    "\n",
    "    df: DataFrame of pregnancy records\n",
    "\n",
    "    returns: list of (rsquared, variable name) pairs\n",
    "    \"\"\"\n",
    "    variables = []\n",
    "    for name in df.columns:\n",
    "        try:\n",
    "            if df[name].var() < 1e-7:\n",
    "                continue\n",
    "\n",
    "            formula = 'totalwgt_lb ~ agepreg + ' + name\n",
    "            formula = formula.encode('ascii')\n",
    "\n",
    "            model = smf.ols(formula, data=df)\n",
    "            if model.nobs < len(df)/2:\n",
    "                continue\n",
    "\n",
    "            results = model.fit()\n",
    "        except (ValueError, TypeError, patsy.PatsyError):\n",
    "            continue\n",
    "        \n",
    "        variables.append((results.rsquared, name))\n",
    "\n",
    "    return variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "variables = GoMining(join)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following functions report the variables with the highest values of $R^2$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def ReadVariables():\n",
    "    \"\"\"Reads Stata dictionary files for NSFG data.\n",
    "\n",
    "    returns: DataFrame that maps variables names to descriptions\n",
    "    \"\"\"\n",
    "    vars1 = thinkstats2.ReadStataDct('2002FemPreg.dct').variables\n",
    "    vars2 = thinkstats2.ReadStataDct('2002FemResp.dct').variables\n",
    "\n",
    "    all_vars = vars1.append(vars2)\n",
    "    all_vars.index = all_vars.name\n",
    "    return all_vars\n",
    "\n",
    "def MiningReport(variables, n=30):\n",
    "    \"\"\"Prints variables with the highest R^2.\n",
    "\n",
    "    t: list of (R^2, variable name) pairs\n",
    "    n: number of pairs to print\n",
    "    \"\"\"\n",
    "    all_vars = ReadVariables()\n",
    "\n",
    "    variables.sort(reverse=True)\n",
    "    for r2, name in variables[:n]:\n",
    "        key = re.sub('_r$', '', name)\n",
    "        try:\n",
    "            desc = all_vars.loc[key].desc\n",
    "            if isinstance(desc, pd.Series):\n",
    "                desc = desc[0]\n",
    "            print(name, r2, desc)\n",
    "        except (KeyError, IndexError):\n",
    "            print(name, r2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some of the variables that do well are not useful for prediction because they are not known ahead of time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "MiningReport(variables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Combining the variables that seem to have the most explanatory power."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>       <td>totalwgt_lb</td>   <th>  R-squared:         </th> <td>   0.060</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.059</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   79.98</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Thu, 28 Feb 2019</td> <th>  Prob (F-statistic):</th> <td>4.86e-113</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>09:59:13</td>     <th>  Log-Likelihood:    </th> <td> -14295.</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>  8781</td>      <th>  AIC:               </th> <td>2.861e+04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>  8773</td>      <th>  BIC:               </th> <td>2.866e+04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     7</td>      <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "            <td></td>              <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>            <td>    6.6303</td> <td>    0.065</td> <td>  102.223</td> <td> 0.000</td> <td>    6.503</td> <td>    6.757</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>C(race)[T.2]</th>         <td>    0.3570</td> <td>    0.032</td> <td>   11.215</td> <td> 0.000</td> <td>    0.295</td> <td>    0.419</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>C(race)[T.3]</th>         <td>    0.2665</td> <td>    0.051</td> <td>    5.175</td> <td> 0.000</td> <td>    0.166</td> <td>    0.367</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>babysex == 1[T.True]</th> <td>    0.2952</td> <td>    0.026</td> <td>   11.216</td> <td> 0.000</td> <td>    0.244</td> <td>    0.347</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>nbrnaliv > 1[T.True]</th> <td>   -1.3783</td> <td>    0.108</td> <td>  -12.771</td> <td> 0.000</td> <td>   -1.590</td> <td>   -1.167</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>paydu == 1[T.True]</th>   <td>    0.1196</td> <td>    0.031</td> <td>    3.861</td> <td> 0.000</td> <td>    0.059</td> <td>    0.180</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>agepreg</th>              <td>    0.0074</td> <td>    0.003</td> <td>    2.921</td> <td> 0.004</td> <td>    0.002</td> <td>    0.012</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>totincr</th>              <td>    0.0122</td> <td>    0.004</td> <td>    3.110</td> <td> 0.002</td> <td>    0.005</td> <td>    0.020</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>398.813</td> <th>  Durbin-Watson:     </th> <td>   1.604</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th> <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td>1388.362</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>          <td>-0.037</td>  <th>  Prob(JB):          </th> <td>3.32e-302</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>      <td> 4.947</td>  <th>  Cond. No.          </th> <td>    221.</td> \n",
       "</tr>\n",
       "</table><br/><br/>Warnings:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:            totalwgt_lb   R-squared:                       0.060\n",
       "Model:                            OLS   Adj. R-squared:                  0.059\n",
       "Method:                 Least Squares   F-statistic:                     79.98\n",
       "Date:                Thu, 28 Feb 2019   Prob (F-statistic):          4.86e-113\n",
       "Time:                        09:59:13   Log-Likelihood:                -14295.\n",
       "No. Observations:                8781   AIC:                         2.861e+04\n",
       "Df Residuals:                    8773   BIC:                         2.866e+04\n",
       "Df Model:                           7                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "========================================================================================\n",
       "                           coef    std err          t      P>|t|      [0.025      0.975]\n",
       "----------------------------------------------------------------------------------------\n",
       "Intercept                6.6303      0.065    102.223      0.000       6.503       6.757\n",
       "C(race)[T.2]             0.3570      0.032     11.215      0.000       0.295       0.419\n",
       "C(race)[T.3]             0.2665      0.051      5.175      0.000       0.166       0.367\n",
       "babysex == 1[T.True]     0.2952      0.026     11.216      0.000       0.244       0.347\n",
       "nbrnaliv > 1[T.True]    -1.3783      0.108    -12.771      0.000      -1.590      -1.167\n",
       "paydu == 1[T.True]       0.1196      0.031      3.861      0.000       0.059       0.180\n",
       "agepreg                  0.0074      0.003      2.921      0.004       0.002       0.012\n",
       "totincr                  0.0122      0.004      3.110      0.002       0.005       0.020\n",
       "==============================================================================\n",
       "Omnibus:                      398.813   Durbin-Watson:                   1.604\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):             1388.362\n",
       "Skew:                          -0.037   Prob(JB):                    3.32e-302\n",
       "Kurtosis:                       4.947   Cond. No.                         221.\n",
       "==============================================================================\n",
       "\n",
       "Warnings:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "\"\"\""
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "formula = ('totalwgt_lb ~ agepreg + C(race) + babysex==1 + '\n",
    "               'nbrnaliv>1 + paydu==1 + totincr')\n",
    "results = smf.ols(formula, data=join).fit()\n",
    "results.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic regression\n",
    "\n",
    "Example: suppose we are trying to predict `y` using explanatory variables `x1` and `x2`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = np.array([0, 1, 0, 1])\n",
    "x1 = np.array([0, 0, 0, 1])\n",
    "x2 = np.array([0, 1, 1, 1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "According to the logit model the log odds for the $i$th element of $y$ is\n",
    "\n",
    "$\\log o = \\beta_0 + \\beta_1 x_1 + \\beta_2 x_2 $\n",
    "\n",
    "So let's start with an arbitrary guess about the elements of $\\beta$:\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "beta = [-1.5, 2.8, 1.1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plugging in the model, we get log odds."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1.5, -0.4, -0.4,  2.4])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_o = beta[0] + beta[1] * x1 + beta[2] * x2\n",
    "log_o"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Which we can convert to odds."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.22313016,  0.67032005,  0.67032005, 11.02317638])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "o = np.exp(log_o)\n",
    "o"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And then convert to probabilities."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.18242552, 0.40131234, 0.40131234, 0.9168273 ])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = o / (o+1)\n",
    "p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The likelihoods of the actual outcomes are $p$ where $y$ is 1 and $1-p$ where $y$ is 0. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.81757448, 0.40131234, 0.59868766, 0.9168273 ])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "likes = np.where(y, p, 1-p)\n",
    "likes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The likelihood of $y$ given $\\beta$ is the product of `likes`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1800933529673034"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "like = np.prod(likes)\n",
    "like"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Logistic regression works by searching for the values in $\\beta$ that maximize `like`.\n",
    "\n",
    "Here's an example using variables in the NSFG respondent file to predict whether a baby will be a boy or a girl."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import first\n",
    "live, firsts, others = first.MakeFrames()\n",
    "live = live[live.prglngth>30]\n",
    "live['boy'] = (live.babysex==1).astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The mother's age seems to have a small effect."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>Logit Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>        <td>boy</td>       <th>  No. Observations:  </th>  <td>  8884</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>               <td>Logit</td>      <th>  Df Residuals:      </th>  <td>  8882</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>               <td>MLE</td>       <th>  Df Model:          </th>  <td>     1</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>          <td>Thu, 28 Feb 2019</td> <th>  Pseudo R-squ.:     </th> <td>6.144e-06</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>              <td>09:59:16</td>     <th>  Log-Likelihood:    </th> <td> -6156.7</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>converged:</th>           <td>True</td>       <th>  LL-Null:           </th> <td> -6156.8</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th> </th>                      <td> </td>        <th>  LLR p-value:       </th>  <td>0.7833</td>  \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "      <td></td>         <th>coef</th>     <th>std err</th>      <th>z</th>      <th>P>|z|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th> <td>    0.0058</td> <td>    0.098</td> <td>    0.059</td> <td> 0.953</td> <td>   -0.185</td> <td>    0.197</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>agepreg</th>   <td>    0.0010</td> <td>    0.004</td> <td>    0.275</td> <td> 0.783</td> <td>   -0.006</td> <td>    0.009</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                           Logit Regression Results                           \n",
       "==============================================================================\n",
       "Dep. Variable:                    boy   No. Observations:                 8884\n",
       "Model:                          Logit   Df Residuals:                     8882\n",
       "Method:                           MLE   Df Model:                            1\n",
       "Date:                Thu, 28 Feb 2019   Pseudo R-squ.:               6.144e-06\n",
       "Time:                        09:59:16   Log-Likelihood:                -6156.7\n",
       "converged:                       True   LL-Null:                       -6156.8\n",
       "                                        LLR p-value:                    0.7833\n",
       "==============================================================================\n",
       "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
       "------------------------------------------------------------------------------\n",
       "Intercept      0.0058      0.098      0.059      0.953      -0.185       0.197\n",
       "agepreg        0.0010      0.004      0.275      0.783      -0.006       0.009\n",
       "==============================================================================\n",
       "\"\"\""
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = smf.logit('boy ~ agepreg', data=live)\n",
    "results = model.fit()\n",
    "results.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here are the variables that seemed most promising."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692944\n",
      "         Iterations 3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>Logit Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>        <td>boy</td>       <th>  No. Observations:  </th>  <td>  8782</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>               <td>Logit</td>      <th>  Df Residuals:      </th>  <td>  8776</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>               <td>MLE</td>       <th>  Df Model:          </th>  <td>     5</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>          <td>Thu, 28 Feb 2019</td> <th>  Pseudo R-squ.:     </th> <td>0.0001440</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>              <td>09:59:16</td>     <th>  Log-Likelihood:    </th> <td> -6085.4</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>converged:</th>           <td>True</td>       <th>  LL-Null:           </th> <td> -6086.3</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th> </th>                      <td> </td>        <th>  LLR p-value:       </th>  <td>0.8822</td>  \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "        <td></td>          <th>coef</th>     <th>std err</th>      <th>z</th>      <th>P>|z|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>    <td>   -0.0301</td> <td>    0.104</td> <td>   -0.290</td> <td> 0.772</td> <td>   -0.234</td> <td>    0.173</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>C(race)[T.2]</th> <td>   -0.0224</td> <td>    0.051</td> <td>   -0.439</td> <td> 0.660</td> <td>   -0.122</td> <td>    0.077</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>C(race)[T.3]</th> <td>   -0.0005</td> <td>    0.083</td> <td>   -0.005</td> <td> 0.996</td> <td>   -0.163</td> <td>    0.162</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>agepreg</th>      <td>   -0.0027</td> <td>    0.006</td> <td>   -0.484</td> <td> 0.629</td> <td>   -0.014</td> <td>    0.008</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>hpagelb</th>      <td>    0.0047</td> <td>    0.004</td> <td>    1.112</td> <td> 0.266</td> <td>   -0.004</td> <td>    0.013</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>birthord</th>     <td>    0.0050</td> <td>    0.022</td> <td>    0.227</td> <td> 0.821</td> <td>   -0.038</td> <td>    0.048</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                           Logit Regression Results                           \n",
       "==============================================================================\n",
       "Dep. Variable:                    boy   No. Observations:                 8782\n",
       "Model:                          Logit   Df Residuals:                     8776\n",
       "Method:                           MLE   Df Model:                            5\n",
       "Date:                Thu, 28 Feb 2019   Pseudo R-squ.:               0.0001440\n",
       "Time:                        09:59:16   Log-Likelihood:                -6085.4\n",
       "converged:                       True   LL-Null:                       -6086.3\n",
       "                                        LLR p-value:                    0.8822\n",
       "================================================================================\n",
       "                   coef    std err          z      P>|z|      [0.025      0.975]\n",
       "--------------------------------------------------------------------------------\n",
       "Intercept       -0.0301      0.104     -0.290      0.772      -0.234       0.173\n",
       "C(race)[T.2]    -0.0224      0.051     -0.439      0.660      -0.122       0.077\n",
       "C(race)[T.3]    -0.0005      0.083     -0.005      0.996      -0.163       0.162\n",
       "agepreg         -0.0027      0.006     -0.484      0.629      -0.014       0.008\n",
       "hpagelb          0.0047      0.004      1.112      0.266      -0.004       0.013\n",
       "birthord         0.0050      0.022      0.227      0.821      -0.038       0.048\n",
       "================================================================================\n",
       "\"\"\""
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "formula = 'boy ~ agepreg + hpagelb + birthord + C(race)'\n",
    "model = smf.logit(formula, data=live)\n",
    "results = model.fit()\n",
    "results.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To make a prediction, we have to extract the exogenous and endogenous variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "endog = pd.DataFrame(model.endog, columns=[model.endog_names])\n",
    "exog = pd.DataFrame(model.exog, columns=model.exog_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The baseline prediction strategy is to guess \"boy\".  In that case, we're right almost 51% of the time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.507173764518333"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "actual = endog['boy']\n",
    "baseline = actual.mean()\n",
    "baseline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If we use the previous model, we can compute the number of predictions we get right."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3944.0, 548.0)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict = (results.predict() >= 0.5)\n",
    "true_pos = predict * actual\n",
    "true_neg = (1 - predict) * (1 - actual)\n",
    "sum(true_pos), sum(true_neg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And the accuracy, which is slightly higher than the baseline."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5115007970849464"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc = (sum(true_pos) + sum(true_neg)) / len(actual)\n",
    "acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To make a prediction for an individual, we have to get their information into a `DataFrame`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.513091\n",
       "dtype: float64"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columns = ['agepreg', 'hpagelb', 'birthord', 'race']\n",
    "new = pd.DataFrame([[35, 39, 3, 2]], columns=columns)\n",
    "y = results.predict(new)\n",
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This person has a 51% chance of having a boy (according to the model)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "**Exercise:** Suppose one of your co-workers is expecting a baby and you are participating in an office pool to predict the date of birth. Assuming that bets are placed during the 30th week of pregnancy, what variables could you use to make the best prediction? You should limit yourself to variables that are known before the birth, and likely to be available to the people in the pool."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import first\n",
    "live, firsts, others = first.MakeFrames()\n",
    "live = live[live.prglngth>30]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following are the only variables I found that have a statistically significant effect on pregnancy length."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>        <td>prglngth</td>     <th>  R-squared:         </th> <td>   0.011</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.011</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   34.28</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Thu, 28 Feb 2019</td> <th>  Prob (F-statistic):</th> <td>5.09e-22</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>09:59:18</td>     <th>  Log-Likelihood:    </th> <td> -18247.</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>  8884</td>      <th>  AIC:               </th> <td>3.650e+04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>  8880</td>      <th>  BIC:               </th> <td>3.653e+04</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     3</td>      <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "            <td></td>               <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>             <td>   38.7617</td> <td>    0.039</td> <td> 1006.410</td> <td> 0.000</td> <td>   38.686</td> <td>   38.837</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>birthord == 1[T.True]</th> <td>    0.1015</td> <td>    0.040</td> <td>    2.528</td> <td> 0.011</td> <td>    0.023</td> <td>    0.180</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>race == 2[T.True]</th>     <td>    0.1390</td> <td>    0.042</td> <td>    3.311</td> <td> 0.001</td> <td>    0.057</td> <td>    0.221</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>nbrnaliv > 1[T.True]</th>  <td>   -1.4944</td> <td>    0.164</td> <td>   -9.086</td> <td> 0.000</td> <td>   -1.817</td> <td>   -1.172</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>1587.470</td> <th>  Durbin-Watson:     </th> <td>   1.619</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th>  <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td>6160.751</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>           <td>-0.852</td>  <th>  Prob(JB):          </th> <td>    0.00</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>       <td> 6.707</td>  <th>  Cond. No.          </th> <td>    10.9</td>\n",
       "</tr>\n",
       "</table><br/><br/>Warnings:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:               prglngth   R-squared:                       0.011\n",
       "Model:                            OLS   Adj. R-squared:                  0.011\n",
       "Method:                 Least Squares   F-statistic:                     34.28\n",
       "Date:                Thu, 28 Feb 2019   Prob (F-statistic):           5.09e-22\n",
       "Time:                        09:59:18   Log-Likelihood:                -18247.\n",
       "No. Observations:                8884   AIC:                         3.650e+04\n",
       "Df Residuals:                    8880   BIC:                         3.653e+04\n",
       "Df Model:                           3                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "=========================================================================================\n",
       "                            coef    std err          t      P>|t|      [0.025      0.975]\n",
       "-----------------------------------------------------------------------------------------\n",
       "Intercept                38.7617      0.039   1006.410      0.000      38.686      38.837\n",
       "birthord == 1[T.True]     0.1015      0.040      2.528      0.011       0.023       0.180\n",
       "race == 2[T.True]         0.1390      0.042      3.311      0.001       0.057       0.221\n",
       "nbrnaliv > 1[T.True]     -1.4944      0.164     -9.086      0.000      -1.817      -1.172\n",
       "==============================================================================\n",
       "Omnibus:                     1587.470   Durbin-Watson:                   1.619\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):             6160.751\n",
       "Skew:                          -0.852   Prob(JB):                         0.00\n",
       "Kurtosis:                       6.707   Cond. No.                         10.9\n",
       "==============================================================================\n",
       "\n",
       "Warnings:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "\"\"\""
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import statsmodels.formula.api as smf\n",
    "model = smf.ols('prglngth ~ birthord==1 + race==2 + nbrnaliv>1', data=live)\n",
    "results = model.fit()\n",
    "results.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** The Trivers-Willard hypothesis suggests that for many mammals the sex ratio depends on “maternal condition”; that is, factors like the mother’s age, size, health, and social status. See https://en.wikipedia.org/wiki/Trivers-Willard_hypothesis\n",
    "\n",
    "Some studies have shown this effect among humans, but results are mixed. In this chapter we tested some variables related to these factors, but didn’t find any with a statistically significant effect on sex ratio.\n",
    "\n",
    "As an exercise, use a data mining approach to test the other variables in the pregnancy and respondent files. Can you find any factors with a substantial effect?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692991\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692961\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692849\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692996\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692903\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692724\n",
      "         Iterations 6\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692992\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693010\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692985\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692577\n",
      "         Iterations 6\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.686599\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693022\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692903\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693031\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692963\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692964\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692905\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693092\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693107\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692589\n",
      "         Iterations 6\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693006\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692774\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692843\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692612\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692947\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692817\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692947\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693023\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692967\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692995\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692645\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692555\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693021\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693066\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692911\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692917\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693003\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692861\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692859\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692814\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692389\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692517\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692452\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692701\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692290\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692930\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693003\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692926\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693004\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692930\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692861\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692866\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692866\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692806\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692930\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692989\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692987\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693008\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692985\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693006\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692993\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692871\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692994\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692985\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693001\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692942\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692922\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692986\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692573\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692988\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692817\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692588\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692877\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692652\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692991\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692918\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692984\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692952\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693008\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692893\n",
      "         Iterations 5\n",
      "Warning: Maximum number of iterations has been exceeded.\n",
      "         Current function value: 0.692776\n",
      "         Iterations: 35\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692638\n",
      "         Iterations 6\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/downey/anaconda3/envs/ThinkStats2/lib/python3.7/site-packages/statsmodels/base/model.py:508: ConvergenceWarning: Maximum Likelihood optimization failed to converge. Check mle_retvals\n",
      "  \"Check mle_retvals\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692838\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692971\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692985\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692971\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693003\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692973\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692973\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692810\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693003\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692995\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692979\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693003\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.686305\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693006\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692959\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692991\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692984\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692928\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692910\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693007\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693008\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692881\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692866\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692861\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692866\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692930\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692806\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692808\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692986\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693004\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692985\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693039\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692886\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692957\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692888\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692989\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693006\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692921\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692996\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692907\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692439\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692916\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692951\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692950\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693037\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693021\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693018\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692850\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693056\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692885\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693007\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692963\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692994\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692959\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692319\n",
      "         Iterations 6\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692984\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692841\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693006\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692879\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693007\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692871\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692993\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692997\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692956\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693010\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692780\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693055\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693104\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693022\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693069\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693052\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692988\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693037\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693063\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693052\n",
      "         Iterations 3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693078\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693078\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692964\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692801\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693074\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692959\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692995\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693004\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692911\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692833\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693025\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692823\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692970\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692945\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693055\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692908\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692703\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692853\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692981\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693053\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692991\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692962\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692928\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693009\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692999\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692943\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693049\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692901\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692951\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692843\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692834\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692783\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692799\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692850\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692841\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693012\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692986\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693118\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693128\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692958\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692898\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693003\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693005\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692829\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692999\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692972\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692901\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692929\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692894\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692820\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692973\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692960\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692999\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693007\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692993\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692986\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693002\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692968\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692995\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693028\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692992\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693010\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692733\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692645\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693029\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692842\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692821\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692986\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693012\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692721\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693054\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692742\n",
      "         Iterations 3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693012\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693007\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692958\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692975\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692848\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692942\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692953\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692915\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692931\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692973\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692940\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692992\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693006\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693000\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693008\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693012\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693012\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692999\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692979\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693000\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692984\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692978\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692930\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692829\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692992\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692980\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692971\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693010\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693000\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693000\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692979\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692996\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693008\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693006\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692985\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693010\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693003\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692990\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693012\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693012\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692950\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693037\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693039\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693091\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693065\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692992\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692987\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692840\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692984\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693008\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693002\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693023\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693001\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693016\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693006\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692970\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693006\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692980\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692989\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693007\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692985\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693003\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692994\n",
      "         Iterations 3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692990\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693003\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692985\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692965\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692994\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692818\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692980\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692680\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692674\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692884\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692865\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693007\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692883\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692994\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692941\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692948\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693008\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692917\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692993\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692999\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692968\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692850\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692971\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692844\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692963\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692980\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692984\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692937\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692960\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692970\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693001\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692911\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693002\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692929\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692995\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692845\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692981\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692718\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692973\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692575\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692985\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692673\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692988\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692778\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693018\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692804\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693023\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692752\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693023\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692671\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692986\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692709\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693046\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692650\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693063\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692875\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693098\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692806\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693074\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693058\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693024\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692892\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692959\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692853\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692615\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692979\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693046\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692917\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692977\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692985\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693004\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692985\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692932\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692619\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692779\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692886\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692739\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692662\n",
      "         Iterations 3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692621\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692792\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692926\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692879\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692805\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693009\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692673\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692940\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692903\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693005\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693006\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692994\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692992\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692980\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693001\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692999\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693002\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692941\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692972\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692952\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692977\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692920\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692856\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692955\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692852\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692573\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692976\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692879\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693069\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692955\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692868\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693003\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693018\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693027\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692949\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693012\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692945\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692962\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692971\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693012\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692579\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692949\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692801\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692987\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693094\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693036\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692917\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692971\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693006\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692824\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692862\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692862\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692976\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693009\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693009\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692965\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692967\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692973\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693008\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692916\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693005\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692951\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692962\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692985\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692952\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692946\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692995\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692974\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692992\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693012\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692995\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692994\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692866\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692806\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692989\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692987\n",
      "         Iterations 3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692985\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693008\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693006\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693005\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693007\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693008\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692610\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692904\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692859\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692712\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692948\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692993\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692871\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692883\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692801\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693001\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692994\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692722\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692909\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692561\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692803\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692870\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692755\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692702\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693008\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692963\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692879\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692855\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692837\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692729\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693001\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692990\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692875\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692989\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692813\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692475\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693001\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692833\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692529\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692898\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692640\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692784\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692875\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692782\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692938\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692871\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692973\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692973\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692973\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692973\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692973\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692939\n",
      "         Iterations 18\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693009\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693005\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692976\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693012\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692995\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692958\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693002\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692983\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693010\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692955\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692967\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693009\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692859\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693010\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692925\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692969\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692983\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692894\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692874\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692859\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693010\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692924\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692982\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692914\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692969\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692983\n",
      "         Iterations 4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692695\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692955\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692946\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693009\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692859\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693010\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692925\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692969\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692983\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692955\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692889\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692823\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692968\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692859\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692925\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692969\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692983\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692989\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692997\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693010\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692923\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692990\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692986\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692929\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693001\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693005\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692939\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692990\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692986\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692929\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692991\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692966\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692941\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692978\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692986\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692929\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692973\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692973\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692974\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692973\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692930\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692929\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692933\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693045\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693078\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692739\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693012\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692997\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692750\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692991\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692873\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692830\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693082\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692636\n",
      "         Iterations 3\n",
      "Warning: Maximum number of iterations has been exceeded.\n",
      "         Current function value: 0.692709\n",
      "         Iterations: 35\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692863\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692926\n",
      "         Iterations 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/downey/anaconda3/envs/ThinkStats2/lib/python3.7/site-packages/statsmodels/base/model.py:508: ConvergenceWarning: Maximum Likelihood optimization failed to converge. Check mle_retvals\n",
      "  \"Check mle_retvals\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692726\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692774\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692999\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692861\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692705\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692723\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692803\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692956\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692786\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693007\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692989\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692993\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692952\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693012\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692970\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693009\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 4\n",
      "Warning: Maximum number of iterations has been exceeded.\n",
      "         Current function value: 0.692862\n",
      "         Iterations: 35\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692861\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/downey/anaconda3/envs/ThinkStats2/lib/python3.7/site-packages/statsmodels/base/model.py:508: ConvergenceWarning: Maximum Likelihood optimization failed to converge. Check mle_retvals\n",
      "  \"Check mle_retvals\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693010\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692658\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692789\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693008\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692855\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692855\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692749\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692862\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692862\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692821\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692772\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692687\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692947\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692771\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692755\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693006\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693012\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692949\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692912\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692879\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692950\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692991\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692929\n",
      "         Iterations 4\n",
      "Warning: Maximum number of iterations has been exceeded.\n",
      "         Current function value: 0.692960\n",
      "         Iterations: 35\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693007\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692981\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692975\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692961\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693005\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692995\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693001\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/downey/anaconda3/envs/ThinkStats2/lib/python3.7/site-packages/statsmodels/base/model.py:508: ConvergenceWarning: Maximum Likelihood optimization failed to converge. Check mle_retvals\n",
      "  \"Check mle_retvals\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692957\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692983\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692258\n",
      "         Iterations 5\n",
      "Warning: Maximum number of iterations has been exceeded.\n",
      "         Current function value: 0.692696\n",
      "         Iterations: 35\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692993\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693009\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692853\n",
      "         Iterations 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/downey/anaconda3/envs/ThinkStats2/lib/python3.7/site-packages/statsmodels/base/model.py:508: ConvergenceWarning: Maximum Likelihood optimization failed to converge. Check mle_retvals\n",
      "  \"Check mle_retvals\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692971\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692639\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692917\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692760\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692832\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693028\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692888\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692770\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692992\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692976\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692866\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692992\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692976\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692866\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693010\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692896\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692758\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693081\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692986\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692927\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692827\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693008\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692890\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692941\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692920\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692920\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692982\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693001\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692997\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693008\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693008\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693005\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692956\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692931\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692996\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692996\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692730\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692898\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692879\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692887\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692987\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692986\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692995\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692980\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692849\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692974\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692987\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692957\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693009\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693002\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693013\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692980\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692849\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692974\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692987\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692957\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693009\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693002\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692969\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693005\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692946\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692862\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692905\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692962\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692977\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693010\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693000\n",
      "         Iterations 3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692998\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693012\n",
      "         Iterations 3\n",
      "Warning: Maximum number of iterations has been exceeded.\n",
      "         Current function value: 0.692939\n",
      "         Iterations: 35\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693003\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692831\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692999\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693003\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692997\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692795\n",
      "         Iterations 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/downey/anaconda3/envs/ThinkStats2/lib/python3.7/site-packages/statsmodels/base/model.py:508: ConvergenceWarning: Maximum Likelihood optimization failed to converge. Check mle_retvals\n",
      "  \"Check mle_retvals\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692693\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692457\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692815\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693002\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692989\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693008\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692673\n",
      "         Iterations 5\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692982\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692985\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692986\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692922\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692973\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693011\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693015\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692942\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692921\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693002\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692810\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693003\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692995\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692979\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693014\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.693003\n",
      "         Iterations 3\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692996\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692996\n",
      "         Iterations 4\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.692971\n",
      "         Iterations 3\n"
     ]
    }
   ],
   "source": [
    "# Solution\n",
    "\n",
    "def GoMining(df):\n",
    "    \"\"\"Searches for variables that predict birth weight.\n",
    "\n",
    "    df: DataFrame of pregnancy records\n",
    "\n",
    "    returns: list of (rsquared, variable name) pairs\n",
    "    \"\"\"\n",
    "    df['boy'] = (df.babysex==1).astype(int)\n",
    "    variables = []\n",
    "    for name in df.columns:\n",
    "        try:\n",
    "            if df[name].var() < 1e-7:\n",
    "                continue\n",
    "\n",
    "            formula='boy ~ agepreg + ' + name\n",
    "            model = smf.logit(formula, data=df)\n",
    "            nobs = len(model.endog)\n",
    "            if nobs < len(df)/2:\n",
    "                continue\n",
    "\n",
    "            results = model.fit()\n",
    "        except:\n",
    "            continue\n",
    "\n",
    "        variables.append((results.prsquared, name))\n",
    "\n",
    "    return variables\n",
    "\n",
    "variables = GoMining(join)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "totalwgt_lb 0.009696855253233383\n",
      "birthwgt_lb 0.009274460080281988 BD-3 BIRTHWEIGHT IN POUNDS - 1ST BABY FROM THIS PREGNANCY\n",
      "constat3 0.0010985419170438382 3RD PRIORITY CODE FOR CURRENT CONTRACEPTIVE STATUS\n",
      "lbw1 0.0010519527860076705 LOW BIRTHWEIGHT - BABY 1\n",
      "nplaced 0.001010368752280555 # OF R'S BIO CHILDREN SHE PLACED FOR ADOPTION (BASED ON BPA)\n",
      "fmarout5 0.0009096579032891183 FORMAL MARITAL STATUS AT PREGNANCY OUTCOME\n",
      "rmarout6 0.000818252143711895 INFORMAL MARITAL STATUS AT PREGNANCY OUTCOME - 6 CATEGORIES\n",
      "infever 0.0008115919859909004 EVER USED INFERTILITY SERVICES OF ANY KIND\n",
      "frsteatd 0.0007675331422082321 AGE (IN MOS) WHEN 1ST SUPPLEMENTED - 1ST FROM THIS PREG\n",
      "splstwk1 0.0007334122339932581 IF-1 H/P DOING WHAT LAST WEEK (EMPLOYMENT STATUS) 1ST MENTION\n",
      "pmarpreg 0.0007245809157658822 WHETHER PREGNANCY ENDED BEFORE R'S 1ST MARRIAGE (PREMARITALLY)\n",
      "usefstp 0.0007122387685902787 EF-3 USE METHOD AT FIRST SEX WITH 1ST PARTNER IN PAST 12 MONTHS?\n",
      "outcom02 0.0007015744602576479 OUTCOME OF PREGNANCY - 2ND\n",
      "nummult34 0.0006606172426639745 NUMBER OF METHODS REPORTED IN (OCT 2001)\n",
      "coh1dur 0.0006550110146368304 DURATION (IN MONTHS) OF R'S FIRST COHABITATION\n",
      "brnout_r 0.0006438582787924307\n",
      "brnout 0.0006438582787924307\n",
      "bpa_bdscheck1 0.0006384992730465999 WHETHER 1ST LIVEBORN BABY FROM THIS PREGNANCY WAS BPA OR BDS\n",
      "nummult41 0.000624852395994635 NUMBER OF METHODS REPORTED IN (MAY 2002)\n",
      "agepreg_i 0.0006221097878729154 AGEPREG IMPUTATION FLAG\n",
      "educmom 0.0005903653895802385 MOTHER'S (OR MOTHER-FIGURE'S) EDUCATION\n",
      "marout03 0.0005883792792801268 FORMAL MARITAL STATUS WHEN PREGNANCY ENDED - 3RD\n",
      "abort12 0.0005779076671296179 FA-3B RECEIVED ABORTION LAST 12 MONTHS\n",
      "p1yhsage 0.0005631662500977797 CI-6 PARTNER'S AGE AT 1ST SEX-1ST REPORTED PARTNER IN LAST 12 MOS\n",
      "numfirsm1 0.0005601782001567468 TOTAL NUMBER OF RESPONSES IN EB-1 FIRSMETH - PRESCRIPTION METHODS\n",
      "lbw1_i 0.0005499451546340239 LBW1 IMPUTATION FLAG\n",
      "agecon_i 0.0005303754891523571 AGECON IMPUTATION FLAG\n",
      "mar1con1_i 0.0005217496825273837 MAR1CON1 IMPUTATION FLAG\n",
      "nummult35 0.0005188253090530059 NUMBER OF METHODS REPORTED IN (NOV 2001)\n",
      "nummult39 0.0005146121683266003 NUMBER OF METHODS REPORTED IN (MAR 2002)\n"
     ]
    }
   ],
   "source": [
    "# Solution\n",
    "\n",
    "#Here are the 30 variables that yield the highest pseudo-R^2 values.\n",
    "\n",
    "MiningReport(variables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.691874\n",
      "         Iterations 4\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>Logit Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>        <td>boy</td>       <th>  No. Observations:  </th>  <td>  8884</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>               <td>Logit</td>      <th>  Df Residuals:      </th>  <td>  8880</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>               <td>MLE</td>       <th>  Df Model:          </th>  <td>     3</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>          <td>Thu, 28 Feb 2019</td> <th>  Pseudo R-squ.:     </th> <td>0.001653</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>              <td>09:59:57</td>     <th>  Log-Likelihood:    </th> <td> -6146.6</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>converged:</th>           <td>True</td>       <th>  LL-Null:           </th> <td> -6156.8</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th> </th>                      <td> </td>        <th>  LLR p-value:       </th> <td>0.0001432</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "            <td></td>               <th>coef</th>     <th>std err</th>      <th>z</th>      <th>P>|z|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>             <td>   -0.1805</td> <td>    0.118</td> <td>   -1.534</td> <td> 0.125</td> <td>   -0.411</td> <td>    0.050</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>fmarout5 == 5[T.True]</th> <td>    0.1582</td> <td>    0.049</td> <td>    3.217</td> <td> 0.001</td> <td>    0.062</td> <td>    0.255</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>infever == 1[T.True]</th>  <td>    0.2194</td> <td>    0.065</td> <td>    3.374</td> <td> 0.001</td> <td>    0.092</td> <td>    0.347</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>agepreg</th>               <td>    0.0050</td> <td>    0.004</td> <td>    1.172</td> <td> 0.241</td> <td>   -0.003</td> <td>    0.013</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                           Logit Regression Results                           \n",
       "==============================================================================\n",
       "Dep. Variable:                    boy   No. Observations:                 8884\n",
       "Model:                          Logit   Df Residuals:                     8880\n",
       "Method:                           MLE   Df Model:                            3\n",
       "Date:                Thu, 28 Feb 2019   Pseudo R-squ.:                0.001653\n",
       "Time:                        09:59:57   Log-Likelihood:                -6146.6\n",
       "converged:                       True   LL-Null:                       -6156.8\n",
       "                                        LLR p-value:                 0.0001432\n",
       "=========================================================================================\n",
       "                            coef    std err          z      P>|z|      [0.025      0.975]\n",
       "-----------------------------------------------------------------------------------------\n",
       "Intercept                -0.1805      0.118     -1.534      0.125      -0.411       0.050\n",
       "fmarout5 == 5[T.True]     0.1582      0.049      3.217      0.001       0.062       0.255\n",
       "infever == 1[T.True]      0.2194      0.065      3.374      0.001       0.092       0.347\n",
       "agepreg                   0.0050      0.004      1.172      0.241      -0.003       0.013\n",
       "=========================================================================================\n",
       "\"\"\""
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Solution\n",
    "\n",
    "# Eliminating variables that are not known during pregnancy and \n",
    "# others that are fishy for various reasons, here's the best model I could find:\n",
    "\n",
    "formula='boy ~ agepreg + fmarout5==5 + infever==1'\n",
    "model = smf.logit(formula, data=join)\n",
    "results = model.fit()\n",
    "results.summary() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** If the quantity you want to predict is a count, you can use Poisson regression, which is implemented in StatsModels with a function called `poisson`. It works the same way as `ols` and `logit`. As an exercise, let’s use it to predict how many children a woman has born; in the NSFG dataset, this variable is called `numbabes`.\n",
    "\n",
    "Suppose you meet a woman who is 35 years old, black, and a college graduate whose annual household income exceeds $75,000. How many children would you predict she has born?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Solution\n",
    "\n",
    "# I used a nonlinear model of age.\n",
    "\n",
    "join.numbabes.replace([97], np.nan, inplace=True)\n",
    "join['age2'] = join.age_r**2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 1.677002\n",
      "         Iterations 7\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>Poisson Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>     <td>numbabes</td>     <th>  No. Observations:  </th>   <td>  8884</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>              <td>Poisson</td>     <th>  Df Residuals:      </th>   <td>  8877</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>               <td>MLE</td>       <th>  Df Model:          </th>   <td>     6</td>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>          <td>Thu, 28 Feb 2019</td> <th>  Pseudo R-squ.:     </th>   <td>0.03686</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>              <td>09:59:57</td>     <th>  Log-Likelihood:    </th>  <td> -14898.</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>converged:</th>           <td>True</td>       <th>  LL-Null:           </th>  <td> -15469.</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th> </th>                      <td> </td>        <th>  LLR p-value:       </th> <td>3.681e-243</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "        <td></td>          <th>coef</th>     <th>std err</th>      <th>z</th>      <th>P>|z|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>    <td>   -1.0324</td> <td>    0.169</td> <td>   -6.098</td> <td> 0.000</td> <td>   -1.364</td> <td>   -0.701</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>C(race)[T.2]</th> <td>   -0.1401</td> <td>    0.015</td> <td>   -9.479</td> <td> 0.000</td> <td>   -0.169</td> <td>   -0.111</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>C(race)[T.3]</th> <td>   -0.0991</td> <td>    0.025</td> <td>   -4.029</td> <td> 0.000</td> <td>   -0.147</td> <td>   -0.051</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>age_r</th>        <td>    0.1556</td> <td>    0.010</td> <td>   15.006</td> <td> 0.000</td> <td>    0.135</td> <td>    0.176</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>age2</th>         <td>   -0.0020</td> <td>    0.000</td> <td>  -13.102</td> <td> 0.000</td> <td>   -0.002</td> <td>   -0.002</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>totincr</th>      <td>   -0.0187</td> <td>    0.002</td> <td>   -9.830</td> <td> 0.000</td> <td>   -0.022</td> <td>   -0.015</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>educat</th>       <td>   -0.0471</td> <td>    0.003</td> <td>  -16.076</td> <td> 0.000</td> <td>   -0.053</td> <td>   -0.041</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                          Poisson Regression Results                          \n",
       "==============================================================================\n",
       "Dep. Variable:               numbabes   No. Observations:                 8884\n",
       "Model:                        Poisson   Df Residuals:                     8877\n",
       "Method:                           MLE   Df Model:                            6\n",
       "Date:                Thu, 28 Feb 2019   Pseudo R-squ.:                 0.03686\n",
       "Time:                        09:59:57   Log-Likelihood:                -14898.\n",
       "converged:                       True   LL-Null:                       -15469.\n",
       "                                        LLR p-value:                3.681e-243\n",
       "================================================================================\n",
       "                   coef    std err          z      P>|z|      [0.025      0.975]\n",
       "--------------------------------------------------------------------------------\n",
       "Intercept       -1.0324      0.169     -6.098      0.000      -1.364      -0.701\n",
       "C(race)[T.2]    -0.1401      0.015     -9.479      0.000      -0.169      -0.111\n",
       "C(race)[T.3]    -0.0991      0.025     -4.029      0.000      -0.147      -0.051\n",
       "age_r            0.1556      0.010     15.006      0.000       0.135       0.176\n",
       "age2            -0.0020      0.000    -13.102      0.000      -0.002      -0.002\n",
       "totincr         -0.0187      0.002     -9.830      0.000      -0.022      -0.015\n",
       "educat          -0.0471      0.003    -16.076      0.000      -0.053      -0.041\n",
       "================================================================================\n",
       "\"\"\""
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Solution\n",
    "\n",
    "formula='numbabes ~ age_r + age2 + age3 + C(race) + totincr + educat'\n",
    "formula='numbabes ~ age_r + age2 + C(race) + totincr + educat'\n",
    "model = smf.poisson(formula, data=join)\n",
    "results = model.fit()\n",
    "results.summary() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can predict the number of children for a woman who is 35 years old, black, and a college\n",
    "graduate whose annual household income exceeds $75,000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    2.496802\n",
       "dtype: float64"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Solution\n",
    "\n",
    "columns = ['age_r', 'age2', 'age3', 'race', 'totincr', 'educat']\n",
    "new = pd.DataFrame([[35, 35**2, 35**3, 1, 14, 16]], columns=columns)\n",
    "results.predict(new)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise:** If the quantity you want to predict is categorical, you can use multinomial logistic regression, which is implemented in StatsModels with a function called `mnlogit`. As an exercise, let’s use it to guess whether a woman is married, cohabitating, widowed, divorced, separated, or never married; in the NSFG dataset, marital status is encoded in a variable called `rmarital`.\n",
    "\n",
    "Suppose you meet a woman who is 25 years old, white, and a high school graduate whose annual household income is about $45,000. What is the probability that she is married, cohabitating, etc?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 1.084053\n",
      "         Iterations 8\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>MNLogit Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>     <td>rmarital</td>     <th>  No. Observations:  </th>  <td>  8884</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>              <td>MNLogit</td>     <th>  Df Residuals:      </th>  <td>  8849</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>               <td>MLE</td>       <th>  Df Model:          </th>  <td>    30</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>          <td>Thu, 28 Feb 2019</td> <th>  Pseudo R-squ.:     </th>  <td>0.1682</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>              <td>09:59:58</td>     <th>  Log-Likelihood:    </th> <td> -9630.7</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>converged:</th>           <td>True</td>       <th>  LL-Null:           </th> <td> -11579.</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th> </th>                      <td> </td>        <th>  LLR p-value:       </th>  <td> 0.000</td> \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "   <th>rmarital=2</th>     <th>coef</th>     <th>std err</th>      <th>z</th>      <th>P>|z|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>    <td>    9.0156</td> <td>    0.805</td> <td>   11.199</td> <td> 0.000</td> <td>    7.438</td> <td>   10.593</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>C(race)[T.2]</th> <td>   -0.9237</td> <td>    0.089</td> <td>  -10.418</td> <td> 0.000</td> <td>   -1.097</td> <td>   -0.750</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>C(race)[T.3]</th> <td>   -0.6179</td> <td>    0.136</td> <td>   -4.536</td> <td> 0.000</td> <td>   -0.885</td> <td>   -0.351</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>age_r</th>        <td>   -0.3635</td> <td>    0.051</td> <td>   -7.150</td> <td> 0.000</td> <td>   -0.463</td> <td>   -0.264</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>age2</th>         <td>    0.0048</td> <td>    0.001</td> <td>    6.103</td> <td> 0.000</td> <td>    0.003</td> <td>    0.006</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>totincr</th>      <td>   -0.1310</td> <td>    0.012</td> <td>  -11.337</td> <td> 0.000</td> <td>   -0.154</td> <td>   -0.108</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>educat</th>       <td>   -0.1953</td> <td>    0.019</td> <td>  -10.424</td> <td> 0.000</td> <td>   -0.232</td> <td>   -0.159</td>\n",
       "</tr>\n",
       "<tr>\n",
       "   <th>rmarital=3</th>     <th>coef</th>     <th>std err</th>      <th>z</th>      <th>P>|z|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>    <td>    2.9570</td> <td>    3.020</td> <td>    0.979</td> <td> 0.328</td> <td>   -2.963</td> <td>    8.877</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>C(race)[T.2]</th> <td>   -0.4411</td> <td>    0.237</td> <td>   -1.863</td> <td> 0.062</td> <td>   -0.905</td> <td>    0.023</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>C(race)[T.3]</th> <td>    0.0591</td> <td>    0.336</td> <td>    0.176</td> <td> 0.860</td> <td>   -0.600</td> <td>    0.718</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>age_r</th>        <td>   -0.3177</td> <td>    0.177</td> <td>   -1.798</td> <td> 0.072</td> <td>   -0.664</td> <td>    0.029</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>age2</th>         <td>    0.0064</td> <td>    0.003</td> <td>    2.528</td> <td> 0.011</td> <td>    0.001</td> <td>    0.011</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>totincr</th>      <td>   -0.3258</td> <td>    0.032</td> <td>  -10.175</td> <td> 0.000</td> <td>   -0.389</td> <td>   -0.263</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>educat</th>       <td>   -0.0991</td> <td>    0.048</td> <td>   -2.050</td> <td> 0.040</td> <td>   -0.194</td> <td>   -0.004</td>\n",
       "</tr>\n",
       "<tr>\n",
       "   <th>rmarital=4</th>     <th>coef</th>     <th>std err</th>      <th>z</th>      <th>P>|z|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>    <td>   -3.5238</td> <td>    1.205</td> <td>   -2.924</td> <td> 0.003</td> <td>   -5.886</td> <td>   -1.162</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>C(race)[T.2]</th> <td>   -0.3213</td> <td>    0.093</td> <td>   -3.445</td> <td> 0.001</td> <td>   -0.504</td> <td>   -0.139</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>C(race)[T.3]</th> <td>   -0.7706</td> <td>    0.171</td> <td>   -4.509</td> <td> 0.000</td> <td>   -1.106</td> <td>   -0.436</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>age_r</th>        <td>    0.1155</td> <td>    0.071</td> <td>    1.626</td> <td> 0.104</td> <td>   -0.024</td> <td>    0.255</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>age2</th>         <td>   -0.0007</td> <td>    0.001</td> <td>   -0.701</td> <td> 0.483</td> <td>   -0.003</td> <td>    0.001</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>totincr</th>      <td>   -0.2276</td> <td>    0.012</td> <td>  -19.621</td> <td> 0.000</td> <td>   -0.250</td> <td>   -0.205</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>educat</th>       <td>    0.0667</td> <td>    0.017</td> <td>    3.995</td> <td> 0.000</td> <td>    0.034</td> <td>    0.099</td>\n",
       "</tr>\n",
       "<tr>\n",
       "   <th>rmarital=5</th>     <th>coef</th>     <th>std err</th>      <th>z</th>      <th>P>|z|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>    <td>   -2.8963</td> <td>    1.305</td> <td>   -2.220</td> <td> 0.026</td> <td>   -5.453</td> <td>   -0.339</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>C(race)[T.2]</th> <td>   -1.0407</td> <td>    0.104</td> <td>  -10.038</td> <td> 0.000</td> <td>   -1.244</td> <td>   -0.837</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>C(race)[T.3]</th> <td>   -0.5661</td> <td>    0.156</td> <td>   -3.635</td> <td> 0.000</td> <td>   -0.871</td> <td>   -0.261</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>age_r</th>        <td>    0.2411</td> <td>    0.079</td> <td>    3.038</td> <td> 0.002</td> <td>    0.086</td> <td>    0.397</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>age2</th>         <td>   -0.0035</td> <td>    0.001</td> <td>   -2.977</td> <td> 0.003</td> <td>   -0.006</td> <td>   -0.001</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>totincr</th>      <td>   -0.2932</td> <td>    0.015</td> <td>  -20.159</td> <td> 0.000</td> <td>   -0.322</td> <td>   -0.265</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>educat</th>       <td>   -0.0174</td> <td>    0.021</td> <td>   -0.813</td> <td> 0.416</td> <td>   -0.059</td> <td>    0.025</td>\n",
       "</tr>\n",
       "<tr>\n",
       "   <th>rmarital=6</th>     <th>coef</th>     <th>std err</th>      <th>z</th>      <th>P>|z|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th>    <td>    8.0533</td> <td>    0.814</td> <td>    9.890</td> <td> 0.000</td> <td>    6.457</td> <td>    9.649</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>C(race)[T.2]</th> <td>   -2.1871</td> <td>    0.080</td> <td>  -27.211</td> <td> 0.000</td> <td>   -2.345</td> <td>   -2.030</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>C(race)[T.3]</th> <td>   -1.9611</td> <td>    0.138</td> <td>  -14.188</td> <td> 0.000</td> <td>   -2.232</td> <td>   -1.690</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>age_r</th>        <td>   -0.2127</td> <td>    0.052</td> <td>   -4.122</td> <td> 0.000</td> <td>   -0.314</td> <td>   -0.112</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>age2</th>         <td>    0.0019</td> <td>    0.001</td> <td>    2.321</td> <td> 0.020</td> <td>    0.000</td> <td>    0.003</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>totincr</th>      <td>   -0.2945</td> <td>    0.012</td> <td>  -25.320</td> <td> 0.000</td> <td>   -0.317</td> <td>   -0.272</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>educat</th>       <td>   -0.0742</td> <td>    0.018</td> <td>   -4.169</td> <td> 0.000</td> <td>   -0.109</td> <td>   -0.039</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                          MNLogit Regression Results                          \n",
       "==============================================================================\n",
       "Dep. Variable:               rmarital   No. Observations:                 8884\n",
       "Model:                        MNLogit   Df Residuals:                     8849\n",
       "Method:                           MLE   Df Model:                           30\n",
       "Date:                Thu, 28 Feb 2019   Pseudo R-squ.:                  0.1682\n",
       "Time:                        09:59:58   Log-Likelihood:                -9630.7\n",
       "converged:                       True   LL-Null:                       -11579.\n",
       "                                        LLR p-value:                     0.000\n",
       "================================================================================\n",
       "  rmarital=2       coef    std err          z      P>|z|      [0.025      0.975]\n",
       "--------------------------------------------------------------------------------\n",
       "Intercept        9.0156      0.805     11.199      0.000       7.438      10.593\n",
       "C(race)[T.2]    -0.9237      0.089    -10.418      0.000      -1.097      -0.750\n",
       "C(race)[T.3]    -0.6179      0.136     -4.536      0.000      -0.885      -0.351\n",
       "age_r           -0.3635      0.051     -7.150      0.000      -0.463      -0.264\n",
       "age2             0.0048      0.001      6.103      0.000       0.003       0.006\n",
       "totincr         -0.1310      0.012    -11.337      0.000      -0.154      -0.108\n",
       "educat          -0.1953      0.019    -10.424      0.000      -0.232      -0.159\n",
       "--------------------------------------------------------------------------------\n",
       "  rmarital=3       coef    std err          z      P>|z|      [0.025      0.975]\n",
       "--------------------------------------------------------------------------------\n",
       "Intercept        2.9570      3.020      0.979      0.328      -2.963       8.877\n",
       "C(race)[T.2]    -0.4411      0.237     -1.863      0.062      -0.905       0.023\n",
       "C(race)[T.3]     0.0591      0.336      0.176      0.860      -0.600       0.718\n",
       "age_r           -0.3177      0.177     -1.798      0.072      -0.664       0.029\n",
       "age2             0.0064      0.003      2.528      0.011       0.001       0.011\n",
       "totincr         -0.3258      0.032    -10.175      0.000      -0.389      -0.263\n",
       "educat          -0.0991      0.048     -2.050      0.040      -0.194      -0.004\n",
       "--------------------------------------------------------------------------------\n",
       "  rmarital=4       coef    std err          z      P>|z|      [0.025      0.975]\n",
       "--------------------------------------------------------------------------------\n",
       "Intercept       -3.5238      1.205     -2.924      0.003      -5.886      -1.162\n",
       "C(race)[T.2]    -0.3213      0.093     -3.445      0.001      -0.504      -0.139\n",
       "C(race)[T.3]    -0.7706      0.171     -4.509      0.000      -1.106      -0.436\n",
       "age_r            0.1155      0.071      1.626      0.104      -0.024       0.255\n",
       "age2            -0.0007      0.001     -0.701      0.483      -0.003       0.001\n",
       "totincr         -0.2276      0.012    -19.621      0.000      -0.250      -0.205\n",
       "educat           0.0667      0.017      3.995      0.000       0.034       0.099\n",
       "--------------------------------------------------------------------------------\n",
       "  rmarital=5       coef    std err          z      P>|z|      [0.025      0.975]\n",
       "--------------------------------------------------------------------------------\n",
       "Intercept       -2.8963      1.305     -2.220      0.026      -5.453      -0.339\n",
       "C(race)[T.2]    -1.0407      0.104    -10.038      0.000      -1.244      -0.837\n",
       "C(race)[T.3]    -0.5661      0.156     -3.635      0.000      -0.871      -0.261\n",
       "age_r            0.2411      0.079      3.038      0.002       0.086       0.397\n",
       "age2            -0.0035      0.001     -2.977      0.003      -0.006      -0.001\n",
       "totincr         -0.2932      0.015    -20.159      0.000      -0.322      -0.265\n",
       "educat          -0.0174      0.021     -0.813      0.416      -0.059       0.025\n",
       "--------------------------------------------------------------------------------\n",
       "  rmarital=6       coef    std err          z      P>|z|      [0.025      0.975]\n",
       "--------------------------------------------------------------------------------\n",
       "Intercept        8.0533      0.814      9.890      0.000       6.457       9.649\n",
       "C(race)[T.2]    -2.1871      0.080    -27.211      0.000      -2.345      -2.030\n",
       "C(race)[T.3]    -1.9611      0.138    -14.188      0.000      -2.232      -1.690\n",
       "age_r           -0.2127      0.052     -4.122      0.000      -0.314      -0.112\n",
       "age2             0.0019      0.001      2.321      0.020       0.000       0.003\n",
       "totincr         -0.2945      0.012    -25.320      0.000      -0.317      -0.272\n",
       "educat          -0.0742      0.018     -4.169      0.000      -0.109      -0.039\n",
       "================================================================================\n",
       "\"\"\""
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Solution\n",
    "\n",
    "# Here's the best model I could find.\n",
    "\n",
    "formula='rmarital ~ age_r + age2 + C(race) + totincr + educat'\n",
    "model = smf.mnlogit(formula, data=join)\n",
    "results = model.fit()\n",
    "results.summary() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make a prediction for a woman who is 25 years old, white, and a high\n",
    "school graduate whose annual household income is about $45,000."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.750028</td>\n",
       "      <td>0.126397</td>\n",
       "      <td>0.001564</td>\n",
       "      <td>0.033403</td>\n",
       "      <td>0.021485</td>\n",
       "      <td>0.067122</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5\n",
       "0  0.750028  0.126397  0.001564  0.033403  0.021485  0.067122"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Solution\n",
    "\n",
    "# This person has a 75% chance of being currently married, \n",
    "# a 13% chance of being \"not married but living with opposite \n",
    "# sex partner\", etc.\n",
    "\n",
    "columns = ['age_r', 'age2', 'race', 'totincr', 'educat']\n",
    "new = pd.DataFrame([[25, 25**2, 2, 11, 12]], columns=columns)\n",
    "results.predict(new)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
